# -*- coding: utf-8 -*-
"""
spark  将数据输入到文件中
"""

from pyspark import SparkConf, SparkContext
import os

os.environ['PYSPARK_PYTHON'] = "F:/projects/PycharmProjects/myprojects/pythonProject/venv/Scripts/python.exe"
os.environ['HADOOP_HOME'] = "D:/dev/hadoop/hadoop-3.1.3"

conf = SparkConf().setMaster("local[*]").setAppName("test_spark_app")
# 修改为一个分区
conf.set("spark.default.parallelism", "1")

sc = SparkContext(conf=conf)

rdd = sc.parallelize([1, 2, 3, 4, 5])
# 修改为一个分区
# rdd = sc.parallelize([1, 2, 3, 4, 5], 1)
# rdd = sc.parallelize([1, 2, 3, 4, 5], numSlices=1)

rdd.saveAsTextFile("F:/projects/PycharmProjects/myprojects/pythonProject/day08/output")

sc.stop()
